package puppy.graph;

import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;

import org.apache.commons.collections15.Transformer;

import util.math.LogProb;



import algorithms.Constants;

public class DeliciousNodeTransformer implements
		Transformer<DeliciousNode, Double> {

	/**
	 * @param args
	 */

	////Hashtable<String, Integer> totals_urls = null;
	//Hashtable<String, Integer> totals_tags = null;
	Hashtable<String, Float> query = null;
	Hashtable<String, Float> query_topics =null;
	HashSet<String> n_grams = null;
	Hashtable<String, Float> n_grams_freq = null;

	// parameter used for smoothing of query probability
	double query_lamda = 0.9;
	double query_lamda_ngram = 0.9999;
	double query_total = 0.0;
	double diri_lamda = 3000;
	double n_gram_weight = 1;

	double top_n = 0.0;

	double ngram_extender_top_n1 = 3;

	/*
	 * 
	 * parameter to experiment
	 */

	public CreateGraphDirected graph = null;
	AgeAggregator agg = null;
	private CreateLM lm;
	private HashSet<String> absorving_nodes;
	
	public static double max_kl=-10000;
	public static double min_kl=100000;
	

	// calculate p(tag) or p(url)

	public DeliciousNodeTransformer(
			Hashtable<String, Float> query, AgeAggregator agg,
			CreateGraphDirected graph, CreateLM lm, String q_text,
			double n_gram_weight, int mimimum_length_ngram) {

		this.query = query;
		this.agg = agg;

		this.graph = graph;
		this.lm = lm;
		
		this.n_gram_weight = n_gram_weight;

		this.n_grams = graph.query_phraser.getTagMatches(q_text,
				mimimum_length_ngram, false);
		// addNgramsTopNQueryTerms(query, mimimum_length_ngram, false);
		
		q_text = q_text.replace(" ", "_");

		// Hashtable<String, Float> query1 =
		// generatePossibleTokensNGram(q_text);//borrar esto
		n_grams_freq = assignNgramProb(n_grams, query);

		Iterator<String> iter = n_grams.iterator();
		while (iter.hasNext()) {
			iter.next();
			// System.out.println(iter.next());

		}
		// System.exit(0);

		// create extended query n-grams

	}

	
	
	/**
	 * Topics constructor
	 * 
	 * @param query
	 * @param query_topics
	 * @param agg
	 * @param graph
	 * @param lm
	 * @param q_text
	 * @param n_gram_weight
	 * @param mimimum_length_ngram
	 */
	public DeliciousNodeTransformer(
			Hashtable<String, Float> query,Hashtable<String, Float> query_topics, AgeAggregator agg,
			CreateGraphDirected graph, CreateLM lm, String q_text,
			double n_gram_weight, int mimimum_length_ngram,HashSet<String> absorving_nodes) {

		this.absorving_nodes= absorving_nodes;
		this.query = query;
		this.query_topics=query_topics;
		this.agg = agg;

		this.graph = graph;
		this.lm = lm;

		this.n_gram_weight = n_gram_weight;

		this.n_grams = graph.query_phraser.getTagMatches(q_text,
				mimimum_length_ngram, false);
		// addNgramsTopNQueryTerms(query, mimimum_length_ngram, false);

		q_text = q_text.replace(" ", "_");

		// Hashtable<String, Float> query1 =
		// generatePossibleTokensNGram(q_text);//borrar esto
		n_grams_freq = assignNgramProb(n_grams, query);

		Iterator<String> iter = n_grams.iterator();
		while (iter.hasNext()) {
			iter.next();
			// System.out.println(iter.next());

		}
		// System.exit(0);

		// create extended query n-grams

	}
	public void addNgramsTopNQueryTerms(Hashtable<String, Float> query,
			int minimum_length_n_gram, boolean onlyOriginal) {

		Enumeration<String> keys = query.keys();
		while (keys.hasMoreElements()) {

			String q_text = keys.nextElement();

			HashSet<String> temp = graph.query_phraser.getTagMatches(q_text,
					minimum_length_n_gram, onlyOriginal);

			n_grams.addAll(temp);

		}

	}
	
	public double calculateBackgroundProb(DeliciousNode node) {

		double total = 0.0;
		
		total = Constants.TOTAL_TAG_KIDS;
		/*if (!node.isTag()) {

			total = Constants.TOTAL_TAG_ADULTS;

		}

*/		
		double n = node.getProb_adults() + 1;
	
		return ((double) n / (double) total);

	}

	public double calculateKL(DeliciousNode node) {

		
		if(node.isTopic()) return 1;
		
		double fore = calculateForegroundProb(node);
		double back = calculateBackgroundProb(node);

		double kl = fore * Math.log(back) - fore * Math.log(fore);
		//System.out.println(kl);
		kl = Math.exp(kl);
		if(kl>max_kl) max_kl=kl;
		
		if(kl<min_kl) min_kl= kl;
		//normalize
		
		kl = (kl- Constants.MIN_KL)/(Constants.MAX_KL-Constants.MIN_KL );
		//System.out.println("Normzalie:"+kl);
		
		if(kl>1 ){ 
		//	System.out.println("KL divergency error " + kl);
			//System.exit(0);
			}
		if(kl<0){
			//System.out.println("KL divergency error " + kl);
			//System.exit(0);
			//kl=0.0000000000001;
		}
		
		//System.out.println(max_kl  + "\t"+ min_kl);
		
		return kl;

	}

	public double calculateForegroundProb(DeliciousNode node) {
		double total = 0.0;
		total = Constants.TOTAL_TAG_KIDS;
		
		
		/*if (!node.isTag()) {

			// total= agg.getAggregate(totals_urls);
			total = Constants.TOTAL_URL_KIDS;

		}
*/
		double n = node.getProb()+1;
		
		return ((double) n / (double) total);

	}

	

	
	public double calculateNodePrior(DeliciousNode node) {

		
		if (absorving_nodes!=null && absorving_nodes.contains(node.getName())){
			
			//System.out.println(node.getName() + "\t  0 query probability");
			//System.exit(1);
			return Math.log(0);
			
		}
		
		double total = 0.0;
		
		double n = node.getProb();
		
		if (node.isTopic()) {
			//System.out.println("Found topic node " + "\t"+ node.getName() + "\t"+ n);
			//System.exit(1);
			return  Math.log(node.getProb());
		}
	
		//
	//	double kl = calculateKL(node);
		
		 
		// return Math.log(kl);
	//	return (double)n/(double)Constants.TOTAL_NODE_MASS;
		return Math.log((double) n / (double) Constants.TOTAL_TAG_KIDS) ;
		
	//	return calculateQueryProb(node);
	}

	/**
	 * prior in query
	 * 
	 * This probability is smooted using diricthlet with the prior probability
	 * as background
	 * 
	 * Special weight is included for bigram form tags star_wars_lego --> query:
	 * star_wars
	 * 
	 * 
	 * @param node
	 * @return
	 */
	public double calculateQueryProb(DeliciousNode node) {

		
		
		
		/* if node is absorving node
		 * 
		 */
		if (absorving_nodes!=null && absorving_nodes.contains(node.getName())){
			
			
			
				return Math.log(0);
			
		}
		
		
		/*
		 * For topic nodes we have a special case 
		 * 
		 *   we return the probability of  p(z|q)
		 *   base on the prediction of the topic model
		 *   
		 * 
		 */
		
		
		
		
		if(node.isTopic()){
			//System.out.println(query_topics + "\t"+ node.getName());
		//	System.out.println("in topic node " + node.getName() + "\t"+  query_topics.get(node.getName()));
			
		//	System.out.println("found topic as query");
		
			return 0.0;
			//return  Math.log(query_topics.get(node.getName()));
		}
		
		
		double prior = calculateNodePrior(node);
		prior  = Math.exp(prior);
		if (!node.isTag()) {
			
			return Math.log( (1-query_lamda)*prior);

		} else {
		
			double kl= calculateKL(node)*10;
			kl = Math.log(kl);
			
			double prob_adults= Math.log(calculateBackgroundProb(node));
			double prob_kids= Math.log(calculateForegroundProb(node));
			
			String tag = node.getName();

			if (query.containsKey(tag)) {

				Float t = query.get(tag);
				// System.out.println("tag in query:" + query.get(tag) + "\t"+
				// tag);

				return  Math.log(t * query_lamda +  (1 - query_lamda) * prior) -prob_kids ;

			} else if (n_grams_freq.containsKey(tag)) {

				Float t = n_grams_freq.get(tag);
				// System.out.println(t + "\t"+ tag + "\tngramsss");
				return Math.log(t * query_lamda  + (1 - query_lamda) * Math.exp(prior)
						* n_gram_weight) -prob_kids;
				
			} else {

				return  Math.log((1 - query_lamda) * prior) -prob_kids;

			}
		}
	}

	/**
	 * Calculates p(q|tag)
	 * 
	 * using LM of Delicious
	 * 
	 * @param query
	 * @param tag
	 * @return
	 */
	public double calculateQueryConditionalProb(DeliciousNode tag) {

		double r = 0d;

		Enumeration<String> keys = query.keys();
		double n = tag.getProb();

	
		
		while (keys.hasMoreElements()) {

			String q = keys.nextElement();
			//System.out.println(  q + "\t"+graph.nodes.get(q) );
			double prior_q = calculateNodePrior(graph.nodes.get(q));
			double qt = getCoOccuranceFreqComplex(tag.getName(), q);

			double q_weight = query.get(q);
			
		/*	if(tag.getName().length()<q.length()){
				
				n= n+ agg.getAggregate(graph.nodes.get(q).freq_type);
			}else{
				n= n+ agg.getAggregate(graph.nodes.get(q).freq_type);
				String complement =extractTagComplement(q,tag.getName());
				n =  n + getCoOccuranceFreq(complement, q);
			}
			
			if(qt>n){
				
				qt=n;
			}*/
			
			
			//fix cases for complex tags 
			if(qt>n){
				
				qt=n;
			}
			
			
			double p = ((qt + (diri_lamda)* LogProb.restore_prob(prior_q))/ (n + diri_lamda)) *q_weight;
			r = r+p;
					
			if (tag.getName().equals("star_wars")) {
				//System.out.println(q + "\t  tag:" + tag.getName() + "\t" + qt+ "\t" + r);

				//System.out.println(prior_q + "\t q_weight:" + q_weight	+ "\t n:" + n + "\t" + (qt + (diri_lamda) * prior_q)/ (n + diri_lamda) * q_weight);
			}

		}

		return r;

	}

	public String extractTagComplement(String x, String y) {
		String a = x;
		String b = y;
		if (x.length() >= y.length()) { // tag: star b: star_wars
			a = y;
			b = x;
		}

		// c(star, wars)
		String complement = null;//

		if (b.startsWith(a)) {
			complement = b.replace(a, "");

			complement = complement.replaceFirst("_", "");

		} else if (b.endsWith(a)) {
			complement = b.replace(a, "");
			complement = complement.substring(0, complement.length() - 1);

		} else {
			complement = b.replace(a, "");
			complement = complement.replace("__", "_");

		}
		return complement;

	}

	// tag a ; query b
	public double getCoOccuranceFreqComplex(String x, String y) {

		int freq = 0;

		if (x.equals(y)) {

			DeliciousNode node = graph.nodes.get(x);
			double n =node.getProb();
			return n;

		} else {
			String a = x;
			String b = y;
			if (x.length() >= y.length()) { // tag: star b: star_wars
				a = y;
				b = x;
			}
			DeliciousNode aa = graph.nodes.get(a);
			DeliciousNode bb = graph.nodes.get(b);

			// c(star, wars)
			String complement = null;//

			if (b.startsWith(a)) {
				complement = b.replace(a, "");

				complement = complement.replaceFirst("_", "");

			} else if (b.endsWith(a)) {
				complement = b.replace(a, "");
				complement = complement.substring(0, complement.length() - 1);

			} else {
				complement = b.replace(a, "");
				complement = complement.replace("__", "_");

			}
		//	System.out.println(x  + "\t"+ aa + "\t" + y + "\t"+ bb);
			double n = bb.getProb();
			freq += getCoOccuranceFreq(complement, a);
			freq += getCoOccuranceFreq(a, b);
			freq += n;
			return freq;
		}

	}
	
	// tag a ; query b
	public double getCoOccuranceFreq(String a, String b) {

		int freq = 0;

		
		DeliciousNode aa = graph.nodes.get(a);
		DeliciousNode bb = graph.nodes.get(b);
		if (aa == null || bb == null) {

			return 0;

		}

		if (lm.lm.containsKey(aa.getName())) {

			 Hashtable<String, DeliciousEdgeSimple> temp = lm.lm.get(aa.getName());

			if (temp.containsKey(bb.getName())) {

				freq += agg.getAggregate(temp.get(bb.getName()).freq);
			}
		} else if (lm.lm.containsKey(bb.getName())) {

			 Hashtable<String, DeliciousEdgeSimple> temp  = lm.lm.get(bb.getName());
			if (temp.containsKey(aa.getName())) {
				freq += agg.getAggregate(temp.get(aa.getName()).freq);

			}
		}

		return freq;
	}

	public double calculateQueryProb_temp(DeliciousNode node) {
		double query_lamda_ngram_l = Math.log(query_lamda_ngram);
		if (!node.isTag()) {
			// System.out.println("aqui" + "\t"+ node.getName());
			// return calculateNodePrior(node);
			return 0;
		} else {
			Hashtable<String, Float> h = generatePossibleTokensNGram(node
					.getName());
			Enumeration<String> keys = h.keys();
			float prob_sum = 0;
			// System.out.println("Query prob for :  " + node.getName());
			while (keys.hasMoreElements()) {

				String k = keys.nextElement();

				if (query.containsKey(k) && k.equals("star_wars")) {
					// System.out.println("matched:" + k + "\t" + node.getName()
					// + "\t"+ query);

					double query_prob = Math.log(query.get(k));

					double first_component = query_prob + query_lamda_ngram
							+ prob_sum;// prob_sum*(query_prob*(query_lamda_ngram)
					double second_component = calculateNodePrior(node)
							+ Math.log((1 - query_lamda_ngram)); // calculateNodePrior(node)*(1-query_lamda_ngram)
					prob_sum = (float) LogProb.sumProbLog(first_component,
							second_component, true);
				} else {

					double second_component = calculateNodePrior(node)
							+ Math.log((1 - query_lamda_ngram));
					prob_sum = (float) second_component;
				}
			}

			return prob_sum;
		}
	}

	public double calculateQueryProbq(DeliciousNode node) {

		if (!node.isTag()) {
			// System.out.println("aqui" + "\t"+ node.getName());
			// return calculateNodePrior(node);
			return 0;
		} else {
			Hashtable<String, Float> h = generatePossibleTokensNGram(node
					.getName());
			Enumeration<String> keys = h.keys();
			float prob_sum = 1;
			// System.out.println("Query prob for :  " + node.getName());
			String matched = isInQueryAsToken(query, node.getName());
			if (matched != null) {
				// System.out.println("matched:" + k + "\t" + node.getName() +
				// "\t"+ query);

				float query_prob = query.get(matched);
				prob_sum = (float) (prob_sum * (query_prob
						* (query_lamda_ngram) + calculateNodePrior(node)
						* (1 - query_lamda_ngram)));
			} else {

				prob_sum = (float) (prob_sum * calculateNodePrior(node) * (1 - query_lamda_ngram));
			}

			return prob_sum;
		}
	}

	/**
	 * Returns true if tag is in target
	 * 
	 * e.g star_wars <==> star
	 * 
	 * star_wars_lego <==> lego
	 * 
	 * @param target
	 * @param tag
	 * @return
	 */
	public boolean isInTargetAsToken(String target, String tag) {

		if (target.endsWith("_" + tag))
			return true;

		if (target.startsWith(tag + "_"))
			return true;

		if (target.contains("_" + tag + "_"))
			return true;

		if (tag.endsWith("_" + target))
			return true;

		if (tag.startsWith(target + "_"))
			return true;

		if (tag.contains("_" + target + "_"))
			return true;

		return false;
	}

	public String isInQueryAsToken(Hashtable<String, Float> q, String tag) {

		Enumeration<String> keys = q.keys();
		while (keys.hasMoreElements()) {

			String q_term = keys.nextElement();
			if (isInTargetAsToken(tag, q_term))
				return q_term;
		}

		return null;
	}

	public Hashtable<String, Float> generatePossibleTokensNGram(
			String query_text) {
		Hashtable<String, Float> h = new Hashtable<String, Float>();

		String ngram = query_text.replace(" ", "_");
		String t[] = ngram.split("_");

		if (graph.nodes.containsKey(ngram)) {

			float prior = (float) calculateNodePrior(graph.nodes.get(ngram));
			h.put(ngram, (float) (prior * (1f - query_lamda_ngram)));

		}

		for (int i = 0; i < t.length; i++) {

			if (graph.nodes.containsKey(t[i])) {
				float prior = (float) calculateNodePrior(graph.nodes.get(t[i]));
				h.put(t[i], (float) (prior * (1f - query_lamda_ngram)));
			}

			if (i + 1 < t.length) {

				String m = t[i] + "_" + t[i + 1];
				if (graph.nodes.containsKey(m)) {

					float prior = (float) calculateNodePrior(graph.nodes.get(m));
					h.put(m, (float) (prior * (1f - query_lamda_ngram)));
				}

			}

			if (i + 2 < t.length) {

				String m = t[i] + "_" + t[i + 1] + "_" + t[i + 2];
				if (graph.nodes.containsKey(m)) {

					float prior = (float) calculateNodePrior(graph.nodes.get(m));
					h.put(m, (float) (prior * (1f - query_lamda_ngram)));
				}

			}

		}

		return h;
	}

	public Hashtable<String, Float> assignNgramProb(HashSet<String> n_grams,
			Hashtable<String, Float> q) {

		Hashtable<String, Float> qq = new Hashtable<String, Float>();

		Iterator<String> iter = n_grams.iterator();
		while (iter.hasNext()) { // for each n_Gram

			String n_gram = iter.next();

			Hashtable<String, Float> possible_terms = generatePossibleTokensNGram(n_gram);

			// iterate for each possible term in n_gram
			Hashtable<String, Float> query_probs = new Hashtable<String, Float>();
			Enumeration<String> iter_ngram = possible_terms.keys();
			
			while (iter_ngram.hasMoreElements()) {

				String possible_ngram = iter_ngram.nextElement();
				Enumeration<String> iter2 = q.keys();
				float freq = 0f;

				DeliciousNode node = graph.nodes.get(possible_ngram);
				float possible_ngram_prior = (float) calculateNodePrior(node);

				while (iter2.hasMoreElements()) { // iterating terms found in
													// query

					String tag = iter2.nextElement();
					float prob_g = 0;
					float possible_ngram_prob = 0;

					if (isInTargetAsToken(possible_ngram, tag)
							|| possible_ngram.equals(tag)) {

						prob_g = q.get(tag);

					}
					float prob_temporal = (float) (prob_g * query_lamda_ngram + (1f - query_lamda_ngram)
							* possible_ngram_prior);

					// updating values for
					if (query_probs.containsKey(possible_ngram)) {
						query_probs.put(possible_ngram, prob_temporal
								+ query_probs.get(possible_ngram));
					} else {
						query_probs.put(possible_ngram, prob_temporal);

					}

				}// query terms iterator

			}// possible ngram iterator

			Enumeration<String> keys = query_probs.keys();
			float total_prob = 1;
			while (keys.hasMoreElements()) {

				String possible_ngram = keys.nextElement();
				float prob = query_probs.get(possible_ngram);
				total_prob = total_prob * prob;
			}
			qq.put(n_gram, total_prob);
		}// each ngram iterator

		return qq;
	}

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String s = "star_wars_lego";
		String t = "wars1";

	}

	@Override
	public Double transform(DeliciousNode input) {
		// TODO Auto-generated method stub

		// double t = calculateQueryProb(input);
		double t = calculateQueryProb(input);
		
		//double t = calculateNodePrior(input);
		
		
		// System.out.println("query cond prob:"+ input.getName() + "\t"+ t);
		return t;
	}

	public void clean() {

		query.clear();
		n_grams.clear();
		n_grams_freq.clear();
		query_topics.clear();
		
		query = null;
		n_grams = null;
		n_grams_freq = null;
		query_topics=null;
		
		agg = null;

	}

}